{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第4步：调整样本的参数：subsample & colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from matplotlib import pyplot\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "import seaborn as sns\n",
    "\n",
    "from numpy import nan as NaN\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filled_Form_N</th>\n",
       "      <th>Filled_Form_Y</th>\n",
       "      <th>Device_Type_Mobile</th>\n",
       "      <th>Device_Type_Web-browser</th>\n",
       "      <th>Mobile_Verified_N</th>\n",
       "      <th>Mobile_Verified_Y</th>\n",
       "      <th>Source_S122</th>\n",
       "      <th>Source_S123</th>\n",
       "      <th>Source_S124</th>\n",
       "      <th>Source_S127</th>\n",
       "      <th>...</th>\n",
       "      <th>Loan_Amount_Submitted</th>\n",
       "      <th>Loan_Tenure_Submitted</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>Disbursed</th>\n",
       "      <th>DOB_month</th>\n",
       "      <th>DOB_year</th>\n",
       "      <th>age</th>\n",
       "      <th>Lead_Creation_Date_month</th>\n",
       "      <th>Lead_Creation_Date_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>620000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.99</td>\n",
       "      <td>3100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1987</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>260000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>33.00</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1993</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>6600.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1978</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1985</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>16.25</td>\n",
       "      <td>7000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1975</td>\n",
       "      <td>44</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Filled_Form_N  Filled_Form_Y  Device_Type_Mobile  Device_Type_Web-browser  \\\n",
       "0              0              1                   1                        0   \n",
       "1              1              0                   1                        0   \n",
       "2              1              0                   0                        1   \n",
       "3              1              0                   0                        1   \n",
       "4              1              0                   0                        1   \n",
       "\n",
       "   Mobile_Verified_N  Mobile_Verified_Y  Source_S122  Source_S123  \\\n",
       "0                  0                  1            1            0   \n",
       "1                  0                  1            1            0   \n",
       "2                  0                  1            1            0   \n",
       "3                  1                  0            1            0   \n",
       "4                  0                  1            1            0   \n",
       "\n",
       "   Source_S124  Source_S127           ...             Loan_Amount_Submitted  \\\n",
       "0            0            0           ...                          620000.0   \n",
       "1            0            0           ...                          260000.0   \n",
       "2            0            0           ...                          100000.0   \n",
       "3            0            0           ...                          200000.0   \n",
       "4            0            0           ...                          300000.0   \n",
       "\n",
       "   Loan_Tenure_Submitted  Interest_Rate  Processing_Fee  Disbursed  DOB_month  \\\n",
       "0                    4.0          13.99          3100.0        0.0          8   \n",
       "1                    4.0          33.00          2000.0        0.0          2   \n",
       "2                    5.0          28.50          6600.0        0.0          2   \n",
       "3                    3.0          28.50          5000.0        0.0          6   \n",
       "4                    5.0          16.25          7000.0        0.0          4   \n",
       "\n",
       "   DOB_year  age  Lead_Creation_Date_month  Lead_Creation_Date_year  \n",
       "0      1987   32                         7                     2015  \n",
       "1      1993   26                         7                     2015  \n",
       "2      1978   41                         7                     2015  \n",
       "3      1985   34                         7                     2015  \n",
       "4      1975   44                         7                     2015  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('FE_X_train.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['Disbursed']\n",
    "\n",
    "train = train.drop([\"Disbursed\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8],\n",
       " 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#subsample\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.2504595091965421,\n",
       " {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       " make_scorer(log_loss, greater_is_better=False, needs_proba=True))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "\n",
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=14,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=2,#调整后\n",
    "        min_child_weight=0.001,#调整后\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'binary:logistic',\n",
    "#         num_class = 9,\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=2, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.best_score_, gsearch3_1.best_params_, gsearch3_1.scorer_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.06372542, 0.05684524, 0.05732608, 0.05833311, 0.05816617,\n",
       "        0.05786433, 0.05386529, 0.06004481, 0.063061  , 0.06346121,\n",
       "        0.0628643 , 0.06292419, 0.05672593, 0.06342273, 0.07367349,\n",
       "        0.08017359, 0.06887956, 0.09495678, 0.08053522, 0.08310642,\n",
       "        0.08505468, 0.08989363, 0.07134914, 0.08372197]),\n",
       " 'std_fit_time': array([0.00967103, 0.00209357, 0.00076854, 0.00079948, 0.00068806,\n",
       "        0.00090003, 0.00063689, 0.00104971, 0.00089403, 0.001094  ,\n",
       "        0.0007615 , 0.00062538, 0.00145602, 0.0006331 , 0.00437166,\n",
       "        0.00724223, 0.00142359, 0.01712238, 0.00552892, 0.0028183 ,\n",
       "        0.00446794, 0.00302626, 0.00242458, 0.0118136 ]),\n",
       " 'mean_score_time': array([0.00293322, 0.00271482, 0.00266066, 0.00265241, 0.00250726,\n",
       "        0.00264239, 0.00249949, 0.0025332 , 0.00305738, 0.00252886,\n",
       "        0.00249043, 0.00253124, 0.00249581, 0.00252848, 0.00282283,\n",
       "        0.00278735, 0.00252929, 0.00338321, 0.00322394, 0.00364008,\n",
       "        0.00317411, 0.00326004, 0.0030344 , 0.00339541]),\n",
       " 'std_score_time': array([4.99867699e-04, 4.81835145e-04, 8.23040274e-05, 3.58564422e-05,\n",
       "        9.63826266e-05, 7.10743340e-05, 9.43425179e-05, 9.52200944e-05,\n",
       "        1.01016435e-03, 9.58683923e-05, 1.11106701e-04, 1.30042105e-04,\n",
       "        1.25461411e-04, 9.44583479e-05, 2.79375200e-04, 3.04053710e-04,\n",
       "        3.38388456e-05, 6.00729558e-04, 5.30176737e-04, 7.99467270e-04,\n",
       "        4.95427185e-04, 5.71458461e-04, 7.35785823e-04, 6.51198049e-04]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.7,\n",
       "                    0.7, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.9, 0.9, 0.9, 0.9,\n",
       "                    0.9, 0.9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6, 0.7,\n",
       "                    0.8, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6,\n",
       "                    0.7, 0.8],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " 'split0_test_score': array([-0.25151613, -0.24984245, -0.25031155, -0.24982558, -0.24971441,\n",
       "        -0.24917554, -0.2506854 , -0.24950697, -0.24800233, -0.24752009,\n",
       "        -0.24781598, -0.2482897 , -0.24886082, -0.25009529, -0.24947508,\n",
       "        -0.2490479 , -0.24777281, -0.2483859 , -0.24736238, -0.24686589,\n",
       "        -0.24818601, -0.24757661, -0.2476607 , -0.24773162]),\n",
       " 'split1_test_score': array([-0.25267745, -0.2502187 , -0.249552  , -0.25023446, -0.249937  ,\n",
       "        -0.2503169 , -0.25201645, -0.25089128, -0.24999648, -0.24947266,\n",
       "        -0.2493103 , -0.24854268, -0.24947394, -0.24905673, -0.24939755,\n",
       "        -0.24946878, -0.24921722, -0.24928428, -0.2487713 , -0.25000494,\n",
       "        -0.24793776, -0.24931893, -0.24863912, -0.24878564]),\n",
       " 'split2_test_score': array([-0.25188284, -0.25096659, -0.25097856, -0.25124142, -0.25129014,\n",
       "        -0.25144744, -0.25328235, -0.25203649, -0.25162058, -0.25196089,\n",
       "        -0.25048809, -0.25027203, -0.25366797, -0.2531047 , -0.25141359,\n",
       "        -0.25050734, -0.25091889, -0.24982949, -0.25193312, -0.25031052,\n",
       "        -0.24984601, -0.25018259, -0.2501469 , -0.25023093]),\n",
       " 'split3_test_score': array([-0.25459042, -0.25350296, -0.25339579, -0.25427701, -0.2546361 ,\n",
       "        -0.25472005, -0.2562616 , -0.25396499, -0.25437817, -0.25387921,\n",
       "        -0.25397369, -0.25445933, -0.25524608, -0.25521861, -0.25401263,\n",
       "        -0.25452501, -0.25366053, -0.25504705, -0.25529178, -0.25458734,\n",
       "        -0.25378357, -0.2536897 , -0.25395683, -0.25416   ]),\n",
       " 'split4_test_score': array([-0.25401716, -0.25336955, -0.25331557, -0.25247501, -0.25207411,\n",
       "        -0.25179148, -0.2529303 , -0.25227587, -0.25201065, -0.25201565,\n",
       "        -0.25145096, -0.25149704, -0.25196869, -0.25127111, -0.25152055,\n",
       "        -0.2510581 , -0.25146393, -0.25126448, -0.25232026, -0.25279927,\n",
       "        -0.25256533, -0.25201997, -0.2518969 , -0.25212064]),\n",
       " 'mean_test_score': array([-0.25293636, -0.25157933, -0.25150997, -0.25161035, -0.25153013,\n",
       "        -0.25149016, -0.25303526, -0.2517349 , -0.25120132, -0.25096928,\n",
       "        -0.25060746, -0.2506118 , -0.25184345, -0.25174948, -0.25116374,\n",
       "        -0.25092137, -0.25060633, -0.25076204, -0.25113529, -0.25091283,\n",
       "        -0.25046289, -0.25055697, -0.25045951, -0.25060516]),\n",
       " 'std_test_score': array([0.00119152, 0.00155867, 0.00157272, 0.00161722, 0.00177996,\n",
       "        0.00185826, 0.00184651, 0.00148567, 0.00212679, 0.00222218,\n",
       "        0.00207555, 0.00225321, 0.00242672, 0.0021953 , 0.00169011,\n",
       "        0.00193927, 0.00200631, 0.00233768, 0.00279833, 0.00263182,\n",
       "        0.00234053, 0.00212201, 0.00226046, 0.00230858]),\n",
       " 'rank_test_score': array([23, 18, 16, 19, 17, 15, 24, 20, 14, 11,  6,  7, 22, 21, 13, 10,  5,\n",
       "         8, 12,  9,  2,  3,  1,  4], dtype=int32),\n",
       " 'split0_train_score': array([-0.25176269, -0.25084279, -0.25045483, -0.25027332, -0.250236  ,\n",
       "        -0.25022393, -0.25263314, -0.25107995, -0.25035477, -0.24959103,\n",
       "        -0.24941511, -0.24969038, -0.25126887, -0.2501038 , -0.24954003,\n",
       "        -0.24944621, -0.24934806, -0.24947067, -0.25025214, -0.24877274,\n",
       "        -0.24897573, -0.24897414, -0.2491132 , -0.2489342 ]),\n",
       " 'split1_train_score': array([-0.2521692 , -0.2506442 , -0.25024498, -0.25075219, -0.24998942,\n",
       "        -0.25060511, -0.25252464, -0.25158393, -0.25025095, -0.25027654,\n",
       "        -0.24998787, -0.24945885, -0.25056432, -0.24984423, -0.24939328,\n",
       "        -0.2495068 , -0.24956209, -0.24956425, -0.25084772, -0.25041135,\n",
       "        -0.24927614, -0.24896071, -0.24899077, -0.24945832]),\n",
       " 'split2_train_score': array([-0.25089648, -0.24999957, -0.24944816, -0.24981618, -0.24973847,\n",
       "        -0.2498116 , -0.25123048, -0.24961893, -0.24931471, -0.24870103,\n",
       "        -0.24872922, -0.2487716 , -0.25015567, -0.24981002, -0.24876475,\n",
       "        -0.24823073, -0.24845192, -0.24834306, -0.24957165, -0.24839485,\n",
       "        -0.24785188, -0.24770495, -0.24814973, -0.24836249]),\n",
       " 'split3_train_score': array([-0.24874151, -0.24762535, -0.24740896, -0.24812399, -0.24819504,\n",
       "        -0.24815767, -0.2491506 , -0.24781907, -0.24784208, -0.24713442,\n",
       "        -0.24697425, -0.2471725 , -0.24836111, -0.24777485, -0.24716995,\n",
       "        -0.24711197, -0.2471766 , -0.24726841, -0.24791052, -0.24676816,\n",
       "        -0.24621854, -0.24634842, -0.246806  , -0.24694129]),\n",
       " 'split4_train_score': array([-0.25090522, -0.24992197, -0.2500348 , -0.25012242, -0.24953165,\n",
       "        -0.24949463, -0.25013239, -0.24959017, -0.24921338, -0.24919546,\n",
       "        -0.2488462 , -0.24878955, -0.24915502, -0.24837003, -0.24872169,\n",
       "        -0.24857771, -0.24869411, -0.24870889, -0.24981361, -0.24940674,\n",
       "        -0.24923728, -0.24920135, -0.24881786, -0.24892746]),\n",
       " 'mean_train_score': array([-0.25089502, -0.24980678, -0.24951835, -0.24981762, -0.24953812,\n",
       "        -0.24965859, -0.25113425, -0.24993841, -0.24939518, -0.2489797 ,\n",
       "        -0.24879053, -0.24877658, -0.249901  , -0.24918058, -0.24871794,\n",
       "        -0.24857468, -0.24864656, -0.24867106, -0.24967913, -0.24875077,\n",
       "        -0.24831191, -0.24823791, -0.24837551, -0.24852475]),\n",
       " 'std_train_score': array([0.00118441, 0.00114749, 0.0011069 , 0.00089915, 0.000712  ,\n",
       "        0.00083888, 0.00135112, 0.00132104, 0.00090607, 0.00105686,\n",
       "        0.00101286, 0.0008802 , 0.00102992, 0.00092974, 0.00084029,\n",
       "        0.00088162, 0.00084039, 0.00083828, 0.00098504, 0.0012038 ,\n",
       "        0.00116795, 0.00108138, 0.00085238, 0.0008643 ])}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # summarize results\n",
    "# print(\"Best: %f using %s\" % (gsearch3_1.best_score_, gsearch3_1.best_params_))\n",
    "# test_means = gsearch3_1.cv_results_[ 'mean_test_score' ]\n",
    "# test_stds = gsearch3_1.cv_results_[ 'std_test_score' ]\n",
    "# train_means = gsearch3_1.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = gsearch3_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# pd.DataFrame(gsearch3_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# # plot results\n",
    "# test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "# train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "# for i, value in enumerate(colsample_bytree):\n",
    "#     pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "# #for i, value in enumerate(min_child_weight):\n",
    "# #    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "# pyplot.legend()\n",
    "# pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "# pyplot.ylabel( 'Log Loss' )\n",
    "# pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.2504595091965421"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.9, 'subsample': 0.7}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.best_params_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
